import time

from dyne.spider.proxy.website import ProxyFactory
from dyne.spider.proxy.browser.agent import useragent
from dyne.spider.tianmao.db_utils import DbUtil

factory = ProxyFactory()
headers = {
    'user-agent': useragent.random()
}


'''
小幻HTTP代理 
www = factory.create('https://ip.ihuan.me/',headers = headers)
'''

'''
快代理
https://www.kuaidaili.com/free/
'''
# www = factory.create('https://www.kuaidaili.com/free',headers = headers)


def doWorker(www):
    dbUtil = DbUtil()
    datas = www.run()
    start_time = int(round(time.time() * 1000))
    for data in datas:
        for item in data:
            dbUtil.insert_proxy(item.getHost(), item.getPort(), item.getScheme(), start_time)
    dbUtil.closeDB()

def start():
    useful_web=[{
     "url":"http://www.66ip.cn/mo.php?sxb=&tqsl=100&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=",
     "charset":"gbk"
    }]
    for website in useful_web:
        www=factory.create(website["url"],website["charset"],headers=headers)
        doWorker(www)

start()






'''
35页左右


89免费代理 http://www.89ip.cn/
www = factory.create('http://www.89ip.cn/',headers = headers)
'''

'''
一次性

http://www.goubanjia.com/
全网代理IP 

'''
# www = factory.create('http://www.goubanjia.com/', headers=headers)


'''
10 页

云代理 http://www.ip3366.net/
# 
'''
# www = factory.create('http://www.ip3366.net/','gbk',headers = headers)



'''
一次性

无忧代理 http://www.data5u.com/
www = factory.create('http://www.data5u.com/',headers = headers)
'''



'''
一次性  

66免费代理网
www = factory.create('http://www.66ip.cn/mo.php?sxb=&tqsl=100&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=',
                     'gbk',
                     headers=headers)
'''